package org.tony.spider.tieba;

import java.io.UnsupportedEncodingException;
import java.net.URLDecoder;
import java.net.URLEncoder;
import java.util.Iterator;
import java.util.Stack;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import org.apache.commons.lang3.StringUtils;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import org.tony.spider.common.FileUtil;
import org.tony.spider.common.GetRequestor;
import org.tony.spider.util.CnStringUtil;

public class TiebaSpider
{

  private ExecutorService               pool    = null;
  private String                        baseUrl = "http://tieba.baidu.com";
  private String                        uri;
  private String                        argName;
  private Stack<Integer>                pages;
  private Integer                       totalePages;
  private Integer                       pageSize;
  private AtomicInteger                 counter;
  private ConcurrentMap<String, String> themes;

  private void init()
  {
    counter = new AtomicInteger();
    pool = Executors.newFixedThreadPool(50);
    GetRequestor getReq = new GetRequestor();
    String responseText = getReq.startGet(baseUrl + uri);
    Document doc = Jsoup.parse(responseText);
    Elements totalPages = doc.select(".pager").select("a[href]");
    if (totalPages != null)
    {
      Element lastPage = totalPages.last();
      String href = lastPage.attr("href");
      String[] array = href.split("&")[1].split("=");
      argName = array[0];
      totalePages = Integer.parseInt(array[1]);
      parsePageSize(totalPages);
      generatePageList();
    }
  }

  private void generatePageList()
  {
    pages = new Stack<Integer>();
    for (int i = totalePages; i > 0; i = i - pageSize)
    {
      pages.push(i);
    }
  }

  public void getThemesList()
  {
    init();
    themes = new ConcurrentHashMap<String, String>();
    GetThemesList gt1 = new GetThemesList(baseUrl + uri);
    pool.execute(gt1);
    while (!pages.empty())
    {
      String url = baseUrl + uri + "&" + argName + "=" + pages.pop();
      GetThemesList gt = new GetThemesList(url);
      counter.incrementAndGet();
      pool.execute(gt);
    }
    pool.shutdown();
    while (true)
    {

      if (counter.get() == 0)
      {
        try
        {
          Thread.sleep(1000);
        }
        catch (InterruptedException e)
        {
        }
        //        for (Iterator<String> key = themes.keySet().iterator(); key.hasNext();)
        //        {
        //          String keyStr = key.next();
        //          System.out.println(keyStr + "," + themes.get(keyStr));
        //        }
        start(themes);
        break;
      }
    }
  }

  private void parsePageSize(Elements pages)
  {
    int firstPage = getPageNum(pages.first());
    int second = getPageNum(pages.get(1));
    pageSize = second - firstPage;
  }

  private int getPageNum(Element ele)
  {
    String href = ele.attr("href");
    String[] array = href.split("&")[1].split("=");
    return array.length == 2 ? Integer.parseInt(array[1]) : 0;
  }

  private class GetThemesList
      implements Runnable
  {
    private String url;

    public GetThemesList(String url)
    {
      super();
      this.url = url;
    }

    public void run()
    {
      GetRequestor getReq = new GetRequestor();
      String content = getReq.startGet(url);
      Document doc = Jsoup.parse(content);
      Elements eles = doc.select("#content_leftList").select("div.th_lz").select("a[href");
      Iterator<Element> iter = eles.iterator();
      while (iter.hasNext())
      {
        Element ele = iter.next();
        String href = ele.attr("href");
        String title = ele.text();
        System.out.println(title);
        if (title.matches(".*\u7B2C.+\u7AE0.*") && !title.contains("图"))
        {
          String newT = StringUtils.deleteWhitespace(title).replaceAll("[\\(|（].+[\\)|）]", "")
              .replaceAll("，|。|！|？|《.*》|【.*】", "");
          Pattern p = Pattern.compile("(.+)(\u7B2C.+\u7AE0.+)");
          Matcher matcher = p.matcher(newT);
          if (matcher.find())
          {
            themes.put(newT, href);
            //System.out.println(matcher.group(2));
          }
        }
      }
      //中问。！，·（）
      counter.decrementAndGet();
    }
  }

  public void start(ConcurrentMap<String, String> themes)
  {
    int notfound = 0;
    Pattern p = Pattern.compile("(.*)(\u7B2C.+\u7AE0.+)");
    Matcher matcher = null;
    try
    {
      Stack<String> s = FileUtil.readFile("d:\\data\\狂枭\\un_download_chapter.data", Stack.class);
      while (!s.empty())
      {
        String orignTitle = URLDecoder.decode(s.pop(), "GB2312");
        String undload = CnStringUtil.removeCnSymbol(orignTitle);
        matcher = p.matcher(undload);
        if (matcher.find())
        {
          undload = matcher.group(2);
        }
        boolean founded = false;
        for (Iterator<String> key = themes.keySet().iterator(); key.hasNext();)
        {
          String keyStr = key.next();
          String title = CnStringUtil.removeCnSymbol(keyStr);
          matcher = p.matcher(title);
          if (matcher.find())
          {
            String themesTitle = matcher.group(2);
            String commTitle = StringUtils.getCommonPrefix(undload, themesTitle);
            if (commTitle.length() > undload.length() - 2)
            {
              founded = true;
              System.out
                  .println("chapter page found:" + orignTitle + "," + this.baseUrl + themes.get(keyStr));
              break;
            }
          }
        }
        if (!founded)
        {
          notfound++;
          System.err.println("chapter not found:" + orignTitle);
        }
      }
      System.out.println("total not found:" + notfound);
    }
    catch (UnsupportedEncodingException e)
    {
    }
  }

  public static void main(String[] args) throws UnsupportedEncodingException
  {
    TiebaSpider ts = new TiebaSpider();
    ts.uri = "/f?kw=" + URLEncoder.encode("狂枭", "GB2312");
    ts.getThemesList();
    //    System.out.println(URLEncoder.encode("&", "GB2312"));
    //    System.out.println(URLDecoder.decode("%CA%A5%D3%FC", "GB2312"));
    //    Stack<String> s = FileUtil.readFile("d:\\data\\狂枭\\un_download_chapter.data", Stack.class);
//    while (!s.empty())
    //    {
    //      String ci = URLDecoder.decode(s.pop(), "GB2312");
    //      System.out.println(ci + "," + CnStringUtil.removeCnSymbol(ci));
    //    }
  }
}
